/*****************************************************************************************
  Project   : The Impact of Unions on Non-union Wage Setting: Threats and Bargaining
  Authors   : David A. Green, Ben M. Sand, Iain G. Snoddy, Jeanne Tschopp
  Date      : August 2025
  Output    : Creates the data for the decomposition used in 
			: Table 3, and Figures 4 and 6
  Notes     : This file has options for:
			: demographic group `1', 
            : whether to create figures`2' (no longer in use, figures created in R, see master file)
			: transition case `3'
			: how many repititions `4'
*****************************************************************************************/


clear
set more off
cap log close

/*

Decomposition after the new model, Nov 10th, 2023

*/

global Use_data "${wd}/decomp_out_${ind}_${area}_`1'_case_`3'.dta"
local case `3'
local figures `2'
local reps `4'


//
// set up transitions data [only needs to be done once]
//


use  $Use_data, clear
keep year2 $area $ind union


joinby year2 detind  union using ${wd}/ipums_ind_trans/transitions_`1'_case_`case'.dta, unmatched(both)
tab _merge
keep if _merge==3
drop _merge

// get base year transition rates
	egen tmp = mean(phi_tt) if year2 == 1, by(job job_2)
	egen tmp2 = mean(tmp), by(job job_2)
	gen sphi_tt = tmp2

save "${wd}/temp_decomp_trans.dta", replace
	
//
// set up shares data
//

use  $Use_data, clear

// ----- > weights for city aggregation

	replace reg_wgt = 0 if reg_wgt == .
	egen temp = sum(reg_wgt) if year == 1 , by($area)
	egen wt = mean(temp), by($area) 
	drop temp


// -----> Shares


	// base size
	egen tmp = sum(reg_wgt) if year2 == 1, by($ind union)
	egen base = mean(tmp), by($ind union)
	drop tmp

	egen tmp = sum(reg_wgt), by(year2 $ind union)
	gen rel = tmp / base
	replace rel = 0 if missing(rel)
	drop tmp

	egen x = mean(rel) if union == 0, by(year2 $ind)
	egen xx = mean(x), by(year2 $ind)
	
	
	egen tmp = mean(reg_wgt) if union == 1 & year2 == 1, by( ${area} $ind)
	egen tmp2 = mean(tmp) if union == 1, by(${area} $ind)

	gen tmp3 = reg_wgt if union == 0
	replace tmp3 = tmp2 * xx if union == 1

	// job share within city
	egen sn_jc = sum(tmp3), by(year $area)
	replace sn_jc = tmp3 / sn_jc
	
	drop tmp*
	
// -----> union premium	


	egen id = group(year2 $area $ind)
	tsset id union
	gen uprem = D.prem

	egen tmp = mean(uprem), by(year $ind)
	replace uprem = tmp
	drop tmp
		
	egen tmp = wtmean(uprem) if year2 == 1, by($area $ind union ) 
	egen tmp2 = mean(tmp), by($area $ind union )
	gen uprem_0 = tmp2
	drop tmp*
	tsset, clear
	
	gen reswage_0 = reswage - uprem + uprem_0 if union == 1
	replace reswage = . if reswage_0 == . & union == 1
	
// ------> threat

// new at city-ind
	merge m:1 ${ind} ${area} year2 using  ${wd}/keyvars/union_elections_${area}_${ind}.dta
	tab _merge
	drop if _merge == 2
	drop _merge
	gen threat  = win_frac_ic_5 
	*drop win_*  cert*  frac_*

	egen tmp = mean(threat) if year2 == 1, by($area $ind union)
	egen threat_var = mean(tmp), by($area $ind union)
	drop tmp*
	
	replace threat = . if threat_var == .

	bysort year detind $area: gen c = !missing(threat) if union == 0
	egen cc = mean(c), by(year detind $area )
	keep if cc == 1 | year ==5
	drop c cc
	
	// save for later merge
	save "${wd}/temp_decomp_shares2.dta", replace
	
	// save for sim
	keep year2 $area $ind union n_jc sn_jc reswage reswage_0 
	rename $ind detind_2
	rename union union_2

	save "${wd}/temp_decomp_shares.dta", replace
	
// combine
use "${wd}/temp_decomp_trans.dta", clear

// merge shares
joinby year2 $area detind_2 union_2 using "${wd}/temp_decomp_shares.dta", unmatched(both)
tab _merge
keep if _merge==3
drop _merge

save "${wd}/rep_three.dta"	, replace
	
	
forval i = 1/`reps' {

use "${wd}/rep_three.dta", clear

if `i' > 1 drop reswage reswage_0

// ------------------------------------
// Full
if `i' == 1 {
	gen     Full = reswage_0 if union_2 == 1
	replace Full = reswage   if union_2 == 0
}
// ------------------------------------
// Threat
if `i' == 1 {
	gen     First = reswage   if union_2 == 1
	replace First = reswage   if union_2 == 0
	}
// ------------------------------------
// Premia
if `i' == 1 {
	gen     Second = reswage   if union_2 == 1
	replace Second = reswage   if union_2 == 0
	}	
// ------------------------------------
// Trans
if `i' == 1 {
	gen     Third = reswage   if union_2 == 1
	replace Third = reswage   if union_2 == 0
	}	
		
if `i' > 1 {
	joinby year2 $area detind_2 union_2 using "${wd}/rep_two.dta", unmatched(both)
	tab _merge
	keep if _merge==3
	drop _merge
	
		
}

replace reswage = . if missing(Full)
gen miss = 1 if missing(reswage) | missing(Full) | missing(First) | missing(Second) 

// Relative costs
gen double temp = phi_tt * phi_ii * n_jc  if !missing(reswage)
egen temp2 = sum( temp ), by(job $area year2)
drop temp
gen double Xi =  (phi_tt * phi_ii) / temp2
drop temp2 


// Relative costs
gen double temp = sphi_tt * phi_ii * sn_jc  if !missing(Full)
egen temp2 = sum( temp ), by(job $area year2)
drop temp
gen double sXi =  (sphi_tt * phi_ii) / temp2 if !missing(Full)
drop temp2 

// outside options
gen double  E_c  = Xi * n_jc * reswage

// cf outside options
gen double E_full   = sXi * sn_jc * Full
gen double E_first  = sXi * sn_jc * First


drop Xi
// Relative costs
gen double temp = phi_tt * phi_ii * n_jc  if !missing(Second)
egen temp2 = sum( temp ), by(job $area year2)
drop temp
gen double Xi =  (phi_tt * phi_ii) / temp2 if !missing(Second)
drop temp2 


gen double E_second = Xi * n_jc * Second

drop Xi
// Relative costs
gen double temp = phi_tt * phi_ii * n_jc  if !missing(Third)
egen temp2 = sum( temp ), by(job $area year2)
drop temp
gen double Xi =  (phi_tt * phi_ii) / temp2  if !missing(Third)
drop temp2 
gen double E_third  = Xi * n_jc * Third

// Go back to job city year obs level
collapse (sum) E* , by(year2 $area detind union) fast

foreach var of varlist E*  {
	replace `var' = . if `var' == 0
}

// Save to merge back into orginal data 
*tempfile temp
save "${wd}/rep_one.dta", replace

*use  $Use_data, clear
use "${wd}/temp_decomp_shares2.dta", clear

merge 1:1 year2 $area $ind union using "${wd}/rep_one.dta"
tab _merge
drop _merge 


// retain union vars
foreach var of varlist E_* {
	gen X = `var' if union == 1
	egen `var'u = mean(X), by(year2 $area $ind)
	drop X	
	}
	
// Inner repitition calculating

// Variables for Decomposition
gen alpha0 = 0.647
gen alpha1 = 0.77
gen alpha  = alpha1 - alpha0

// ------------------------------
// Full CF 
// CF wages [non-union]
gen a = alpha0 * E_c    + threat     * (alpha1 * E_cu    - alpha0 * E_c)    if union == 0
gen b = alpha0 * E_full + threat_var * (alpha1 * E_fullu - alpha0 * E_full) if union == 0
gen c = b - a if union == 0

gen Full = reswage + c if union == 0
gen save_c = c
drop b  c

// ------------------------------
// Threat
gen b = alpha0 * E_first + threat_var * (alpha1 * E_firstu - alpha0 * E_first) if union == 0
gen c = b - a if union == 0

gen First = reswage + c if union == 0

drop b c 

// ------------------------------
// union premia
gen b = alpha0 * E_second + threat_var * (alpha1 * E_secondu - alpha0 * E_second) if union == 0
gen c = b - a if union == 0

gen Second = reswage + c if union == 0

drop b c 
// ------------------------------
// transitions
gen b = alpha0 * E_third + threat_var * (alpha1 * E_thirdu - alpha0 * E_third) if union == 0
gen c = b - a if union == 0

gen Third = reswage + c if union == 0

drop a b c 

// ------------------------------
// Full CF 
// wages [union]
replace alpha1 = .414
gen a = alpha1 * E_c  if union == 1
gen b = alpha1 * E_full if union == 1
gen c = b - a if union == 1


replace Full   = reswage_0 + c if union == 1
replace First  = reswage +  c if union == 1

drop b  c
gen b = alpha1 * E_second if union == 1
gen c = b - a

replace Second = reswage + c if union == 1

drop b c 
gen b = alpha1 * E_third if union == 1
gen c = b - a

replace Third = reswage + c if union == 1

drop a b c 

replace reswage = . if missing(Full) 

if `i' < `reps' {
	
	egen W_obs    = wtmean(reswage) , by(year2 ${area} union) weight(n_jc)
	egen W_full   = wtmean(Full)    , by(year2 ${area} union) weight(sn_jc)
	egen W_first  = wtmean(First)   , by(year2 ${area} union) weight(sn_jc)
	egen W_second = wtmean(Second)  , by(year2 ${area} union) weight(n_jc)
	egen W_third  = wtmean(Third)   , by(year2 ${area} union) weight(n_jc)
	
	keep year ${area} $ind union reswage Full First Second Third

	rename $ind detind_2
	rename union union_2
	
	save "${wd}/rep_two.dta", replace 
	}
	
	
}


// city-year level
	
	egen W_obs    = wtmean(reswage) , by(year2 ${area} union) weight(n_jc)
	egen W_full   = wtmean(Full)    , by(year2 ${area} union) weight(sn_jc)
	egen W_first  = wtmean(First)   , by(year2 ${area} union) weight(sn_jc)
	egen W_second = wtmean(Second)  , by(year2 ${area} union) weight(n_jc)
	egen W_third  = wtmean(Third)   , by(year2 ${area} union) weight(n_jc)

egen tmp = wtmean(union) if year2 == 1, by(year2 ${area}) weight(n_jc)
egen union_share80 = mean(tmp), by(${area})
drop tmp

egen union_share = wtmean(union), by(year2 ${area}) weight(n_jc)

collapse (mean) union_sh* W_* , by(year $area union ) fast

// reshape
reshape wide W_* union_share*  , i( year  ${area} ) j(union) 
drop union_share1 union_share801
rename union_share0 union_share
rename union_share800 union_share80

tempfile temp
save `temp', replace

use  $Use_data, clear

// Weights for aggregation
	replace reg_wgt = 0 if reg_wgt == .
	egen temp = sum(reg_wgt) if year == 1 , by($area)
	egen wt = mean(temp), by($area) 
	drop temp
	
collapse (mean) wt, by(year2 $area)

sort year2 $area
merge 1:1 year2 $area using  `temp'

// collapse
	collapse (mean) W_* union_sh* [aw = wt], by(year2)
	tsset year2, yearly	

// breakdown non-union
	// full obs - full CF
	gen double Full0   = W_full0   - W_obs0
	
	gen double Premia0  = W_full0   - W_first0
	gen double Trans0   = W_first0  - W_second0
	gen double Threat0  = W_second0 - W_obs0 
	
	gen double Spill0   = W_full0   - W_second0 // ie, Prema + trans
	

//////////////////////////////////////////////
// New decomp for the revision
//////////////////////////////////////////////


// Overall observed wage trends
gen observed = W_obs0 * (1 - union_share  ) + W_obs1 * union_share - (W_obs0[1] * (1 - union_share[1]) + W_obs1[1] * union_share[1])

// part due to changing union proportion
gen part1   = (union_share - union_share80) * (W_obs1 - W_obs0)

// part due to changing wage differentials
gen part2   = union_share80 * (W_obs1 - W_obs0 - (W_obs1[1] - W_obs0[1]))

// CF 2 - non-union wage trends (ie, holdign union proportion and differential constant)
gen cf2   = W_obs0 - W_obs0[1] 

// CF 1 - holding union proportion constant.
gen cf1 = part2 + cf2

// Decomposing CF 2 into de-union effect and CF 3	

// Deunion	
gen deunion  =  W_obs0 - W_full0

// CF 3 - full counterfactual
gen cf3      =  W_full0 - W_obs0[1]

// full decomp of compsition effect

gen cf_threat = W_obs0 - W_second0
gen cf_transition = W_second0 -  W_first0
gen cf_premia =  W_first0 - W_full0

if "`figures'" == "yes" {
// referee request... how much of part 2 (changing wage differentials) is explained by model, rather than taken as exog.
gen part2_exp   = union_share80 * ( ( W_full1 - W_obs1[1] ) - ( W_full0 - W_obs0[1] ) )
gen part2_unexp = union_share80 * ( ( W_obs1 - W_full1 ) - ( W_obs0 - W_full0 ) )

gen test = part2_exp + part2_unexp - part2

lab var part2 "Change in Union wage differential"
lab var part2_exp "Explained part"

scatter part2 part2_exp  year if year < 5, c(l l l l l l) s(d o Oh s + ) ///
	ytitle("Change in Log Wage") ///
	xtitle("Year") ///
	xlabel(1 "1980" 2 "1990" 3 "2000" 4 "2010", grid) ///
	ylabel(,grid) color(%50 %50 %50 %50 %50 %50 %50) ///
	legend(size(small) cols(2)  symxsize(6)  textwidth(40) ) ///
	name(wage_d, replace)	 
	graph export $figures/wage_d.pdf, as(pdf) replace	

gen a =  W_obs1 - W_obs1[1]
gen b =  W_full1 - W_obs1[1]
gen c =  W_obs1 - W_full1

gen part2_exp2   = union_share80 * ( ( W_first1 - W_obs1[1] ) - ( W_first0 - W_obs0[1] ) )
gen part2_unexp2 = union_share80 * ( ( W_obs1 - W_first1 ) - ( W_obs0 - W_first0 ) )

gen test2 = part2_exp2 + part2_unexp2 - part2

lab var part2 "Change in Union wage differential"
lab var part2_exp2 "Explained part"

scatter part2 part2_exp2  year if year < 5, c(l l l l l l) s(d o Oh s + ) ///
	ytitle("Change in Log Wage") ///
	xtitle("Year") ///
	xlabel(1 "1980" 2 "1990" 3 "2000" 4 "2010", grid) ///
	ylabel(,grid) color(%50 %50 %50 %50 %50 %50 %50) ///
	legend(size(small) cols(2)  symxsize(6)  textwidth(40) ) ///
	name(wage_d2, replace)	 
	graph export $figures/wage_d2.pdf, as(pdf) replace	



// Figure -- overall wages, union fixed and full decomp trends

lab var observed "Observed (Δ{it:w}{subscript:ct})"
lab var cf1 "Fixed Union Prop. (Δ{it:w}{subscript:ct}{superscript:cf1})"
lab var cf2 "Δ{it:w}{subscript:ct}{superscript:cf1} + (Δ{it:w}{subscript:ct}{superscript:cf2})"
lab var cf3 "Full counterfactual (Δ{it:w}{subscript:ct}{superscript:cf3})"

scatter observed cf1 cf2 cf3  year if year < 5, c(l l l l l l) s(d o Oh s + ) ///
	ytitle("Change in Log Wage") ///
	xtitle("Year") ///
	xlabel(1 "1980" 2 "1990" 3 "2000" 4 "2010", grid) ///
	ylabel(,grid) color(%50 %50 %50 %50 %50 %50 %50) ///
	legend(size(small) cols(2)  symxsize(6)  textwidth(40) lab(3 "Δ{it:w}{subscript:ct}{superscript:cf1} + Fixed premium  (Δ{it:w}{subscript:ct}{superscript:cf2})")) ///
	name(overall, replace)	

graph export $figures/Overall_trends.pdf, as(pdf) replace	
	
	
// full decomp of compsition effect

lab var deunion "Non-Union Spillover"
lab var cf_threat "Union Threat Effect (4a) "
lab var cf_premia "Union Wage Premia Effect (4c) "
lab var cf_transition "Transitions Effect (4b)"

scatter deunion cf_threat cf_transition cf_premia     year if year < 5, c(l l l l l l) s(d o Oh s + ) ///
	ytitle("Change in Log Wage") ///
	xtitle("Year") ///
	xlabel(1 "1980" 2 "1990" 3 "2000" 4 "2010", grid) ///
	ylabel(,grid) color(%50 %50 %50 %50 %50 %50 %50) ///
	legend(size(small)) ///
	name(nonunion, replace)
graph export $figures/breakdown_NU.pdf, as(pdf) replace	



gen line1 = deunion - cf_threat
gen line2 = line1   - cf_transition
gen line3 = line2 - cf_premia

lab var line1 "Without Threat (4) - (4a)"
lab var line2 "Without Threat and Transition (4) - (4a) - (4b)"

scatter deunion line1 line2   year if year < 5, c(l l l l l l) s(d o Oh s + ) ///
	ytitle("Change in Log Wage") yline(0,  lpattern(dash) lcolor(gs10)) ///
	xtitle("Year") ///
	xlabel(1 "1980" 2 "1990" 3 "2000" 4 "2010", grid) ///
	ylabel(,grid) color(%50 %50 %50 %50 %50 %50 %50) ///
	legend(size(small)) ///
	name(nonunion, replace)
graph export $figures/breakdown_NU2.pdf, as(pdf) replace	

		
}

// Save output
save ${wd}/decomp_grp_`1'_case_`3'.dta, replace



















